/**
 * 
 */
package test;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import com.battcn.platform.entity.crow.ProxyIp;
import com.battcn.platform.mapper.crow.ProxyIpMapper;
import com.battcn.platform.service.webmagic.downloader.CrowProxyProvider;
import com.battcn.platform.service.webmagic.pageprocessor.MovieProcessor;
import com.battcn.platform.service.webmagic.pageprocessor.ProxyPoolProcessor1;
import com.battcn.platform.service.webmagic.pipeline.IPSpiderPipeline;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.proxy.Proxy;

/**
 * @author yanglei
 *
 * 2018年1月15日
 */
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class crawltest {
	@Autowired
	ProxyIpMapper proxyIpMapper;
	@Autowired
	IPSpiderPipeline ipSpiderPipeline;

	// 代理ip测试
	public synchronized  void getProxyIP() {
		System.out.println("----------------------getProxyIP----------------------------begin");
		int threadNum=30;
		ExecutorService executorService=Executors.newFixedThreadPool(threadNum);
		Spider.create(new ProxyPoolProcessor1()).addUrl("http://www.xicidaili.com/nn").thread(executorService, threadNum)
				// .addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
				.addPipeline(ipSpiderPipeline).run();
		
		System.out.println("----------------------getProxyIP----------------------------end");
		    
	}
	// 电影爬虫测试
			public synchronized void getMovies() {
				System.out.println("----------------------getMovies----------------------------begin");
				int threadNum=30;
				ExecutorService executorService=Executors.newFixedThreadPool(threadNum);
			
				List<ProxyIp> proxyList = proxyIpMapper.findAllProxies();
				proxyList = proxyList.subList(0, 10);
				List<Proxy> proxies = new ArrayList<>(proxyList.size());
				for (ProxyIp proxyIp : proxyList) {
					proxies.add(new Proxy(proxyIp.getIp(), proxyIp.getPort()));
				}

				HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
				// 设置动态转发代理，使用定制的ProxyProvider
				httpClientDownloader.setProxyProvider(new CrowProxyProvider(proxies));
				// httpClientDownloader.setProxyProvider(CrowProxyProvider.from(new
				// Proxy("forward.xdaili.cn", 80)));
				Spider.create(new MovieProcessor())
						// new PostInfoPageProcessor())
						// .setDownloader(httpClientDownloader)
						.addUrl("http://www.80s.tw/")
						// .addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
						.addPipeline(new ConsolePipeline()).thread(executorService,threadNum).run();
			 	System.out.println("----------------------getMovies----------------------------end");
			}
			@Test
		    public void test(){

				  Spider.create(new testprocess()).addUrl("http://www.80s.tw/dm/21582")
				  .addPipeline(new ConsolePipeline()).thread(1).run();
				 
			/*	String url = "<span class='span_block'> <span class='font_888'>豆瓣评分：</span> <span class='score sc8'></span>8.3 </span>";
				String regex = "(?=<span class=\"score sc8\"></span>).*(?=</span>)";
				// String regex = "(?=</span>).*(?=</span>)";
				System.out.println(getMatcher(regex, url));*/

			}
}
